#pragma once
#include "WebPage.h"
#include "SplitTool.h"
#include "Configuration.h"
#include "simhash/Simhasher.hpp"
#include <unordered_map>

//网页库预处理类
class PageLibPreprocessor{
public:
    PageLibPreprocessor(Configuration *config, SplitTool &splitTool);
    ~PageLibPreprocessor();
    void doProcess(); //执行预处理
private:
    void readInfoFromFile(); //根据配置信息读取网页库和位置偏移库的内容
    void cutRedundantPage(); //对网页库进行去重
    void bulidInvertIndexMap(); //创建倒排索引表
    void storeOnDisk(); //将经过预处理之后的网页库、位置偏移库和倒排索引表写回到磁盘上
private:
    vector<WebPage> _pageLib; //网页库的容器对象
    unordered_map<int, pair<int, int>> _offsetLib; //网页偏移库对象
    unordered_map<string, vector<pair<int, double>>> _invertIndexLib; //倒排索引表对象
    Configuration *_config;
    SplitTool &_splitTool;
};

